{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from scipy import ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename='preprocessed_majority.csv'\n",
    "datapd_0=pd.read_csv(filename, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename='preprocessed_minority.csv'\n",
    "datapd_1=pd.read_csv(filename, index_col=0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Majority class dataframe shape: (16383, 99)\n",
      "Minority class dataframe shape: (625, 99)\n"
     ]
    }
   ],
   "source": [
    "print('Majority class dataframe shape:', datapd_0.shape)\n",
    "print('Minority class dataframe shape:', datapd_1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_feat=datapd_0.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imbalance Ratio: 26.2128\n"
     ]
    }
   ],
   "source": [
    "print('Imbalance Ratio:', datapd_0.shape[0]/datapd_1.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_0=np.asarray(datapd_0)\n",
    "features_1=np.asarray(datapd_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "s=93\n",
    "features_1=np.take(features_1,np.random.RandomState(seed=s).permutation(features_1.shape[0]),axis=0,out=features_1)\n",
    "features_0=np.take(features_0,np.random.RandomState(seed=s).permutation(features_0.shape[0]),axis=0,out=features_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=len(features_1)//3\n",
    "b=len(features_0)//3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_1_min=features_1[0:a]\n",
    "fold_1_maj=features_0[0:b]\n",
    "fold_1_tst=np.concatenate((fold_1_min,fold_1_maj))\n",
    "lab_1_tst=np.concatenate((np.zeros(len(fold_1_min))+1, np.zeros(len(fold_1_maj))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_2_min=features_1[a:2*a]\n",
    "fold_2_maj=features_0[b:2*b]\n",
    "fold_2_tst=np.concatenate((fold_2_min,fold_2_maj))\n",
    "lab_2_tst=np.concatenate((np.zeros(len(fold_1_min))+1, np.zeros(len(fold_1_maj))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_3_min=features_1[2*a:]\n",
    "fold_3_maj=features_0[2*b:]\n",
    "fold_3_tst=np.concatenate((fold_3_min,fold_3_maj))\n",
    "lab_3_tst=np.concatenate((np.zeros(len(fold_3_min))+1, np.zeros(len(fold_3_maj))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_1_trn=np.concatenate((fold_2_min,fold_3_min,fold_2_maj,fold_3_maj))\n",
    "lab_1_trn=np.concatenate((np.zeros(a+len(fold_3_min))+1,np.zeros(b+len(fold_3_maj))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_2_trn=np.concatenate((fold_1_min,fold_3_min,fold_1_maj,fold_3_maj))\n",
    "lab_2_trn=np.concatenate((np.zeros(a+len(fold_3_min))+1,np.zeros(b+len(fold_3_maj))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_3_trn=np.concatenate((fold_2_min,fold_1_min,fold_2_maj,fold_1_maj))\n",
    "lab_3_trn=np.concatenate((np.zeros(2*a)+1,np.zeros(2*b)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_folds_feats=[fold_1_trn,fold_2_trn,fold_3_trn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_folds_feats=[fold_1_tst,fold_2_tst,fold_3_tst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_folds_labels=[lab_1_trn,lab_2_trn,lab_3_trn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_folds_labels=[lab_1_tst,lab_2_tst,lab_3_tst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr(X_train,y_train,X_test,y_test):\n",
    "    from sklearn import metrics\n",
    "    from sklearn.linear_model import LogisticRegression\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import f1_score\n",
    "    from sklearn.metrics import precision_score\n",
    "    from sklearn.metrics import recall_score\n",
    "    from sklearn.metrics import balanced_accuracy_score\n",
    "    logreg = LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial', class_weight={0: 1, 1: 1})\n",
    "    logreg.fit(X_train, y_train)\n",
    "    y_pred= logreg.predict(X_test)\n",
    "    con_mat=confusion_matrix(y_test,y_pred)\n",
    "    bal_acc=balanced_accuracy_score(y_test,y_pred)\n",
    "    tn, fp, fn, tp = con_mat.ravel()\n",
    "    print('tn, fp, fn, tp:', tn, fp, fn, tp)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    precision=precision_score(y_test, y_pred)\n",
    "    recall=recall_score(y_test, y_pred)\n",
    "    print('balanced accuracy_LR:', bal_acc)\n",
    "    print('f1 score_LR:', f1)\n",
    "    print('confusion matrix_LR',con_mat)\n",
    "    return(f1, bal_acc, precision, recall, con_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svm(X_train,y_train,X_test,y_test):\n",
    "    from sklearn import preprocessing\n",
    "    from sklearn import metrics\n",
    "    #from sklearn import svm\n",
    "    from sklearn.svm import LinearSVC\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import f1_score\n",
    "    from sklearn.metrics import precision_score\n",
    "    from sklearn.metrics import recall_score\n",
    "    from sklearn.metrics import balanced_accuracy_score\n",
    "    X_train = preprocessing.scale(X_train)\n",
    "    X_test = preprocessing.scale(X_test)\n",
    "    #svm= svm.SVC(kernel='linear', decision_function_shape='ovo', class_weight={0: 1., 1: 1.},probability=True)\n",
    "    svm= LinearSVC(random_state=0, tol=1e-5)\n",
    "    svm.fit(X_train, y_train)\n",
    "    y_pred= svm.predict(X_test)\n",
    "    con_mat=confusion_matrix(y_test,y_pred)\n",
    "    bal_acc=balanced_accuracy_score(y_test,y_pred)\n",
    "    tn, fp, fn, tp = con_mat.ravel()\n",
    "    print('tn, fp, fn, tp:', tn, fp, fn, tp)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    precision=precision_score(y_test, y_pred)\n",
    "    recall=recall_score(y_test, y_pred)\n",
    "    print('balanced accuracy_SVM:', bal_acc)\n",
    "    print('f1 score_SVM:', f1)\n",
    "    print('confusion matrix_SVM',con_mat)\n",
    "    return( f1, bal_acc, precision, recall, con_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def knn(X_train,y_train,X_test,y_test):\n",
    "    from sklearn import metrics\n",
    "    from sklearn.neighbors import KNeighborsClassifier\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import f1_score\n",
    "    from sklearn.metrics import precision_score\n",
    "    from sklearn.metrics import recall_score   \n",
    "    from sklearn.metrics import balanced_accuracy_score\n",
    "    knn = KNeighborsClassifier(n_neighbors=10)\n",
    "    knn.fit(X_train, y_train)\n",
    "    y_pred= knn.predict(X_test)\n",
    "    con_mat=confusion_matrix(y_test,y_pred)\n",
    "    bal_acc=balanced_accuracy_score(y_test,y_pred)\n",
    "    tn, fp, fn, tp = con_mat.ravel()\n",
    "    print('tn, fp, fn, tp:', tn, fp, fn, tp)\n",
    "    print('balanced accuracy_KNN:', bal_acc)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    precision=precision_score(y_test, y_pred)\n",
    "    recall=recall_score(y_test, y_pred)\n",
    "    print('f1 score_KNN:', f1)\n",
    "    print('confusion matrix_KNN',con_mat)\n",
    "    return(f1, bal_acc, precision, recall, con_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Neb_grps(data,near_neb):\n",
    "    nbrs = NearestNeighbors(n_neighbors=near_neb, algorithm='ball_tree').fit(data)\n",
    "    distances, indices = nbrs.kneighbors(data)\n",
    "    neb_class=[]\n",
    "    for i in (indices):\n",
    "        neb_class.append(i)\n",
    "    return(np.asarray(neb_class)) \n",
    "\n",
    "def LoRAS(data,num_samples,shadow,sigma,num_RACOS,num_afcomb):\n",
    "    np.random.seed(42)\n",
    "    data_shadow=([])\n",
    "    for i in range (num_samples):\n",
    "        c=0\n",
    "        while c<shadow:\n",
    "            data_shadow.append(data[i]+np.random.normal(0,sigma))\n",
    "            c=c+1\n",
    "    data_shadow==np.asarray(data_shadow)\n",
    "    data_shadow_lc=([])\n",
    "    for i in range(num_RACOS):\n",
    "        idx = np.random.randint(shadow*num_samples, size=num_afcomb)\n",
    "        w=np.random.randint(100, size=len(idx))\n",
    "        aff_w=np.asarray(w/sum(w))\n",
    "        data_tsl=np.array(data_shadow)[idx,:]\n",
    "        data_tsl_=np.dot(aff_w, data_tsl)\n",
    "        data_shadow_lc.append(data_tsl_)\n",
    "    return(np.asarray(data_shadow_lc))   \n",
    "\n",
    "def LoRAS_gen(num_samples,shadow,sigma,num_RACOS,num_afcomb):\n",
    "    RACOS_set=[]\n",
    "    for i in range (len(nb_list)):\n",
    "        RACOS_i= LoRAS(features_1_trn[nb_list[i]],num_samples,shadow,sigma,num_RACOS,num_afcomb)\n",
    "        RACOS_set.append(RACOS_i)\n",
    "    LoRAS_set=np.asarray(RACOS_set)\n",
    "    LoRAS_1=np.reshape(LoRAS_set,(len(features_1_trn)*num_RACOS,n_feat))\n",
    "    return(np.concatenate((LoRAS_1,features_1_trn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def OVS(training_data,training_labels,neb):\n",
    "    from imblearn.over_sampling import SMOTE \n",
    "    sm = SMOTE(random_state=62, k_neighbors=neb,  kind='regular',ratio=1)\n",
    "    SMOTE_feat, SMOTE_labels = sm.fit_resample(training_data,training_labels)\n",
    "    smbl1 = SMOTE(random_state=62, k_neighbors=neb,  kind='borderline1',ratio=1)\n",
    "    SMOTE_feat_bl1, SMOTE_labels_bl1 = smbl1.fit_resample(training_data,training_labels)\n",
    "    smbl2 = SMOTE(random_state=62, k_neighbors=neb,  kind='borderline2',ratio=1)\n",
    "    SMOTE_feat_bl2, SMOTE_labels_bl2 = smbl2.fit_resample(training_data,training_labels)\n",
    "    smsvm = SMOTE(random_state=62, k_neighbors=neb,  kind='svm',ratio=1)\n",
    "    SMOTE_feat_svm, SMOTE_labels_svm = smsvm.fit_resample(training_data,training_labels)\n",
    "    from imblearn.over_sampling import ADASYN\n",
    "    ad = ADASYN(random_state=62,n_neighbors=neb,  ratio=1)\n",
    "    ADASYN_feat, ADASYN_labels = ad.fit_resample(training_data,training_labels)\n",
    "    return(SMOTE_feat, SMOTE_labels,SMOTE_feat_bl1, SMOTE_labels_bl1, SMOTE_feat_bl2, SMOTE_labels_bl2,SMOTE_feat_svm, SMOTE_labels_svm,ADASYN_feat, ADASYN_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.ensemble.bagging module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.ensemble.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.ensemble.forest module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "Using TensorFlow backend.\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.utils.testing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.utils. Anything that cannot be imported from sklearn.utils is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:143: FutureWarning: The sklearn.metrics.classification module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5438 23 31 177\n",
      "balanced accuracy_LR: 0.923374927809784\n",
      "f1 score_LR: 0.8676470588235293\n",
      "confusion matrix_LR [[5438   23]\n",
      " [  31  177]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5439 22 30 178\n",
      "balanced accuracy_SVM: 0.9258703322862818\n",
      "f1 score_SVM: 0.8725490196078431\n",
      "confusion matrix_SVM [[5439   22]\n",
      " [  30  178]]\n",
      "tn, fp, fn, tp: 5450 11 42 166\n",
      "balanced accuracy_KNN: 0.8980313199892948\n",
      "f1 score_KNN: 0.8623376623376624\n",
      "confusion matrix_KNN [[5450   11]\n",
      " [  42  166]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5378 83 14 194\n",
      "balanced accuracy_LR: 0.958746813066077\n",
      "f1 score_LR: 0.8\n",
      "confusion matrix_LR [[5378   83]\n",
      " [  14  194]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3326 2135 2 206\n",
      "balanced accuracy_SVM: 0.7997152888312933\n",
      "f1 score_SVM: 0.16163201255394274\n",
      "confusion matrix_SVM [[3326 2135]\n",
      " [   2  206]]\n",
      "tn, fp, fn, tp: 5378 83 11 197\n",
      "balanced accuracy_KNN: 0.9659583515276153\n",
      "f1 score_KNN: 0.8073770491803278\n",
      "confusion matrix_KNN [[5378   83]\n",
      " [  11  197]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5371 90 14 194\n",
      "balanced accuracy_LR: 0.9581059048075162\n",
      "f1 score_LR: 0.7886178861788617\n",
      "confusion matrix_LR [[5371   90]\n",
      " [  14  194]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4040 1421 3 205\n",
      "balanced accuracy_SVM: 0.8626840850506388\n",
      "f1 score_SVM: 0.2235550708833152\n",
      "confusion matrix_SVM [[4040 1421]\n",
      " [   3  205]]\n",
      "tn, fp, fn, tp: 5342 119 7 201\n",
      "balanced accuracy_KNN: 0.972277636527545\n",
      "f1 score_KNN: 0.7613636363636364\n",
      "confusion matrix_KNN [[5342  119]\n",
      " [   7  201]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5258 203 15 193\n",
      "balanced accuracy_LR: 0.9453559681940473\n",
      "f1 score_LR: 0.6390728476821191\n",
      "confusion matrix_LR [[5258  203]\n",
      " [  15  193]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4040 1421 3 205\n",
      "balanced accuracy_SVM: 0.8626840850506388\n",
      "f1 score_SVM: 0.2235550708833152\n",
      "confusion matrix_SVM [[4040 1421]\n",
      " [   3  205]]\n",
      "tn, fp, fn, tp: 5298 163 9 199\n",
      "balanced accuracy_KNN: 0.9634413780231854\n",
      "f1 score_KNN: 0.6982456140350877\n",
      "confusion matrix_KNN [[5298  163]\n",
      " [   9  199]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5358 103 13 195\n",
      "balanced accuracy_LR: 0.9593194927668924\n",
      "f1 score_LR: 0.7707509881422925\n",
      "confusion matrix_LR [[5358  103]\n",
      " [  13  195]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3970 1491 2 206\n",
      "balanced accuracy_SVM: 0.8586788486188779\n",
      "f1 score_SVM: 0.2162729658792651\n",
      "confusion matrix_SVM [[3970 1491]\n",
      " [   2  206]]\n",
      "tn, fp, fn, tp: 5356 105 11 197\n",
      "balanced accuracy_KNN: 0.9639440684292817\n",
      "f1 score_KNN: 0.7725490196078431\n",
      "confusion matrix_KNN [[5356  105]\n",
      " [  11  197]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5330 131 12 196\n",
      "balanced accuracy_LR: 0.9591597058864958\n",
      "f1 score_LR: 0.7327102803738318\n",
      "confusion matrix_LR [[5330  131]\n",
      " [  12  196]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3790 1671 3 205\n",
      "balanced accuracy_SVM: 0.8397945043877566\n",
      "f1 score_SVM: 0.19673704414587334\n",
      "confusion matrix_SVM [[3790 1671]\n",
      " [   3  205]]\n",
      "tn, fp, fn, tp: 5342 119 6 202\n",
      "balanced accuracy_KNN: 0.9746814826813911\n",
      "f1 score_KNN: 0.7637051039697543\n",
      "confusion matrix_KNN [[5342  119]\n",
      " [   6  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5434 27 33 175\n",
      "balanced accuracy_LR: 0.9182010022114857\n",
      "f1 score_LR: 0.8536585365853658\n",
      "confusion matrix_LR [[5434   27]\n",
      " [  33  175]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5444 17 38 170\n",
      "balanced accuracy_SVM: 0.9070973546687702\n",
      "f1 score_SVM: 0.8607594936708862\n",
      "confusion matrix_SVM [[5444   17]\n",
      " [  38  170]]\n",
      "tn, fp, fn, tp: 5455 6 45 163\n",
      "balanced accuracy_KNN: 0.8912775731410139\n",
      "f1 score_KNN: 0.8647214854111406\n",
      "confusion matrix_KNN [[5455    6]\n",
      " [  45  163]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5337 124 13 195\n",
      "balanced accuracy_LR: 0.9573967679912103\n",
      "f1 score_LR: 0.7400379506641367\n",
      "confusion matrix_LR [[5337  124]\n",
      " [  13  195]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3618 1843 1 207\n",
      "balanced accuracy_SVM: 0.8288541651993859\n",
      "f1 score_SVM: 0.183348095659876\n",
      "confusion matrix_SVM [[3618 1843]\n",
      " [   1  207]]\n",
      "tn, fp, fn, tp: 5355 106 7 201\n",
      "balanced accuracy_KNN: 0.9734678947220148\n",
      "f1 score_KNN: 0.7805825242718446\n",
      "confusion matrix_KNN [[5355  106]\n",
      " [   7  201]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5323 138 14 194\n",
      "balanced accuracy_LR: 0.9537111053202428\n",
      "f1 score_LR: 0.7185185185185187\n",
      "confusion matrix_LR [[5323  138]\n",
      " [  14  194]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4139 1322 2 206\n",
      "balanced accuracy_SVM: 0.8741522051469863\n",
      "f1 score_SVM: 0.23732718894009214\n",
      "confusion matrix_SVM [[4139 1322]\n",
      " [   2  206]]\n",
      "tn, fp, fn, tp: 5328 133 6 202\n",
      "balanced accuracy_KNN: 0.9733996661642697\n",
      "f1 score_KNN: 0.7440147329650093\n",
      "confusion matrix_KNN [[5328  133]\n",
      " [   6  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5224 237 6 202\n",
      "balanced accuracy_LR: 0.9638776006085107\n",
      "f1 score_LR: 0.6244204018547141\n",
      "confusion matrix_LR [[5224  237]\n",
      " [   6  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4139 1322 2 206\n",
      "balanced accuracy_SVM: 0.8741522051469863\n",
      "f1 score_SVM: 0.23732718894009214\n",
      "confusion matrix_SVM [[4139 1322]\n",
      " [   2  206]]\n",
      "tn, fp, fn, tp: 5263 198 6 202\n",
      "balanced accuracy_KNN: 0.9674483751919203\n",
      "f1 score_KNN: 0.6644736842105263\n",
      "confusion matrix_KNN [[5263  198]\n",
      " [   6  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5327 134 10 198\n",
      "balanced accuracy_LR: 0.9636927232262336\n",
      "f1 score_LR: 0.7333333333333332\n",
      "confusion matrix_LR [[5327  134]\n",
      " [  10  198]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4125 1336 1 207\n",
      "balanced accuracy_SVM: 0.875274234783711\n",
      "f1 score_SVM: 0.2364363221016562\n",
      "confusion matrix_SVM [[4125 1336]\n",
      " [   1  207]]\n",
      "tn, fp, fn, tp: 5360 101 6 202\n",
      "balanced accuracy_KNN: 0.9763295324891186\n",
      "f1 score_KNN: 0.7906066536203522\n",
      "confusion matrix_KNN [[5360  101]\n",
      " [   6  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5297 164 11 197\n",
      "balanced accuracy_LR: 0.9585421273928415\n",
      "f1 score_LR: 0.6924428822495606\n",
      "confusion matrix_LR [[5297  164]\n",
      " [  11  197]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3966 1495 2 206\n",
      "balanced accuracy_SVM: 0.8583126153282719\n",
      "f1 score_SVM: 0.21581980094290204\n",
      "confusion matrix_SVM [[3966 1495]\n",
      " [   2  206]]\n",
      "tn, fp, fn, tp: 5276 185 5 203\n",
      "balanced accuracy_KNN: 0.9710424795402364\n",
      "f1 score_KNN: 0.6812080536912752\n",
      "confusion matrix_KNN [[5276  185]\n",
      " [   5  203]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:86: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5442 19 30 179\n",
      "balanced accuracy_LR: 0.9264900569413913\n",
      "f1 score_LR: 0.8796068796068797\n",
      "confusion matrix_LR [[5442   19]\n",
      " [  30  179]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5447 14 39 170\n",
      "balanced accuracy_SVM: 0.9054167480761801\n",
      "f1 score_SVM: 0.8651399491094148\n",
      "confusion matrix_SVM [[5447   14]\n",
      " [  39  170]]\n",
      "tn, fp, fn, tp: 5455 6 40 169\n",
      "balanced accuracy_KNN: 0.9037568701597847\n",
      "f1 score_KNN: 0.8802083333333334\n",
      "confusion matrix_KNN [[5455    6]\n",
      " [  40  169]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5354 107 18 191\n",
      "balanced accuracy_LR: 0.9471410585193486\n",
      "f1 score_LR: 0.7534516765285996\n",
      "confusion matrix_LR [[5354  107]\n",
      " [  18  191]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3573 1888 2 207\n",
      "balanced accuracy_SVM: 0.8223531978386979\n",
      "f1 score_SVM: 0.1796875\n",
      "confusion matrix_SVM [[3573 1888]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5336 125 7 202\n",
      "balanced accuracy_KNN: 0.9718087981853053\n",
      "f1 score_KNN: 0.7537313432835822\n",
      "confusion matrix_KNN [[5336  125]\n",
      " [   7  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5341 120 19 190\n",
      "balanced accuracy_LR: 0.943558455827271\n",
      "f1 score_LR: 0.7321772639691714\n",
      "confusion matrix_LR [[5341  120]\n",
      " [  19  190]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4050 1411 2 207\n",
      "balanced accuracy_SVM: 0.8660265177434772\n",
      "f1 score_SVM: 0.22660098522167488\n",
      "confusion matrix_SVM [[4050 1411]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5323 138 7 202\n",
      "balanced accuracy_KNN: 0.9706185399908354\n",
      "f1 score_KNN: 0.7358834244080147\n",
      "confusion matrix_KNN [[5323  138]\n",
      " [   7  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5231 230 14 195\n",
      "balanced accuracy_LR: 0.9454487628236412\n",
      "f1 score_LR: 0.6151419558359621\n",
      "confusion matrix_LR [[5231  230]\n",
      " [  14  195]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4050 1411 2 207\n",
      "balanced accuracy_SVM: 0.8660265177434772\n",
      "f1 score_SVM: 0.22660098522167488\n",
      "confusion matrix_SVM [[4050 1411]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5260 201 7 202\n",
      "balanced accuracy_KNN: 0.9648503656637891\n",
      "f1 score_KNN: 0.6601307189542484\n",
      "confusion matrix_KNN [[5260  201]\n",
      " [   7  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5354 107 15 194\n",
      "balanced accuracy_LR: 0.9543180920121715\n",
      "f1 score_LR: 0.7607843137254902\n",
      "confusion matrix_LR [[5354  107]\n",
      " [  15  194]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4062 1399 2 207\n",
      "balanced accuracy_SVM: 0.8671252176152955\n",
      "f1 score_SVM: 0.22809917355371898\n",
      "confusion matrix_SVM [[4062 1399]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5343 118 7 202\n",
      "balanced accuracy_KNN: 0.972449706443866\n",
      "f1 score_KNN: 0.7637051039697542\n",
      "confusion matrix_KNN [[5343  118]\n",
      " [   7  202]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5303 158 18 191\n",
      "balanced accuracy_LR: 0.9424715840641207\n",
      "f1 score_LR: 0.6845878136200717\n",
      "confusion matrix_LR [[5303  158]\n",
      " [  18  191]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3911 1550 2 207\n",
      "balanced accuracy_SVM: 0.8532999108949146\n",
      "f1 score_SVM: 0.21057985757884026\n",
      "confusion matrix_SVM [[3911 1550]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5271 190 7 202\n",
      "balanced accuracy_KNN: 0.9658575072129559\n",
      "f1 score_KNN: 0.6722129783693843\n",
      "confusion matrix_KNN [[5271  190]\n",
      " [   7  202]]\n"
     ]
    }
   ],
   "source": [
    "LR=[]\n",
    "SVM=[]\n",
    "KNN=[] \n",
    "LR_SM=[]\n",
    "SVM_SM=[]\n",
    "KNN_SM=[]\n",
    "LR_SMBL1=[]\n",
    "SVM_SMBL1=[]\n",
    "KNN_SMBL1=[] \n",
    "LR_SMBL2=[]\n",
    "SVM_SMBL2=[]\n",
    "KNN_SMBL2=[] \n",
    "LR_SMSVM=[]\n",
    "SVM_SMSVM=[]\n",
    "KNN_SMSVM=[] \n",
    "LR_ADA=[]\n",
    "SVM_ADA=[]\n",
    "KNN_ADA=[] \n",
    "\n",
    "i=0\n",
    "while i<3:\n",
    "    SMOTE_feat, SMOTE_labels,SMOTE_feat_bl1, SMOTE_labels_bl1, SMOTE_feat_bl2, SMOTE_labels_bl2,SMOTE_feat_svm, SMOTE_labels_svm,ADASYN_feat, ADASYN_labels=OVS(training_folds_feats[i],training_folds_labels[i],30)\n",
    "    \n",
    "    f1_lr, bal_acc_lr, precision_lr, recall_lr, mat_lr=lr(training_folds_feats[i],training_folds_labels[i],testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR.append([f1_lr, bal_acc_lr, precision_lr, recall_lr])\n",
    "    f1_svm,bal_acc_svm,precision_svm, recall_svm,mat_svm=svm(training_folds_feats[i],training_folds_labels[i],testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM.append([f1_svm,bal_acc_svm,precision_svm, recall_svm])\n",
    "    f1_knn,bal_acc_knn,precision_knn, recall_knn,mat_knn=knn(training_folds_feats[i],training_folds_labels[i],testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN.append([f1_knn,bal_acc_knn,precision_knn, recall_knn])\n",
    "    \n",
    "    f1_lr_SMOTE,bal_acc_lr_SMOTE,precision_lr_SMOTE, recall_lr_SMOTE,mat_lr_SMOTE=lr(SMOTE_feat,SMOTE_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_SM.append([f1_lr_SMOTE,bal_acc_lr_SMOTE,precision_lr_SMOTE, recall_lr_SMOTE])\n",
    "    f1_svm_SMOTE,bal_acc_svm_SMOTE,precision_svm_SMOTE, recall_svm_SMOTE,mat_svm_SMOTE=svm(SMOTE_feat,SMOTE_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_SM.append([f1_svm_SMOTE,bal_acc_svm_SMOTE,precision_svm_SMOTE, recall_svm_SMOTE])\n",
    "    f1_knn_SMOTE,bal_acc_knn_SMOTE,precision_knn_SMOTE, recall_knn_SMOTE,mat_knn_SMOTE=knn(SMOTE_feat,SMOTE_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_SM.append([f1_knn_SMOTE,bal_acc_knn_SMOTE,precision_knn_SMOTE, recall_knn_SMOTE])\n",
    "    \n",
    "    f1_lr_SMOTE_bl1,bal_acc_lr_SMOTE_bl1,precision_lr_SMOTE_bl1, recall_lr_SMOTE_bl1,mat_lr_SMOTE_bl1=lr(SMOTE_feat_bl1,SMOTE_labels_bl1,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_SMBL1.append([f1_lr_SMOTE_bl1,bal_acc_lr_SMOTE_bl1,precision_lr_SMOTE_bl1, recall_lr_SMOTE_bl1])\n",
    "    f1_svm_SMOTE_bl1,bal_acc_svm_SMOTE_bl1,precision_svm_SMOTE_bl1, recall_svm_SMOTE_bl1,mat_svm_SMOTE_bl1=svm(SMOTE_feat_bl1,SMOTE_labels_bl1,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_SMBL1.append([f1_svm_SMOTE_bl1,bal_acc_svm_SMOTE_bl1,precision_svm_SMOTE_bl1, recall_svm_SMOTE_bl1])\n",
    "    f1_knn_SMOTE_bl1,bal_acc_knn_SMOTE_bl1,precision_knn_SMOTE_bl1, recall_knn_SMOTE_bl1,mat_knn_SMOTE_bl1=knn(SMOTE_feat_bl1,SMOTE_labels_bl1,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_SMBL1.append([f1_knn_SMOTE_bl1,bal_acc_knn_SMOTE_bl1,precision_knn_SMOTE_bl1, recall_knn_SMOTE_bl1])\n",
    "    \n",
    "    \n",
    "    f1_lr_SMOTE_bl2,bal_acc_lr_SMOTE_bl2,precision_lr_SMOTE_bl2, recall_lr_SMOTE_bl2,mat_lr_SMOTE_bl2=lr(SMOTE_feat_bl2,SMOTE_labels_bl2,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_SMBL2.append([f1_lr_SMOTE_bl2,bal_acc_lr_SMOTE_bl2,precision_lr_SMOTE_bl2, recall_lr_SMOTE_bl2])\n",
    "    f1_svm_SMOTE_bl2,bal_acc_svm_SMOTE_bl2,precision_svm_SMOTE_bl2, recall_svm_SMOTE_bl2,mat_svm_SMOTE_bl2=svm(SMOTE_feat_bl1,SMOTE_labels_bl1,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_SMBL2.append([f1_svm_SMOTE_bl2,bal_acc_svm_SMOTE_bl2,precision_svm_SMOTE_bl2, recall_svm_SMOTE_bl2])\n",
    "    f1_knn_SMOTE_bl2,bal_acc_knn_SMOTE_bl2,precision_knn_SMOTE_bl2, recall_knn_SMOTE_bl2,mat_knn_SMOTE_bl2=knn(SMOTE_feat_bl2,SMOTE_labels_bl2,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_SMBL2.append([f1_knn_SMOTE_bl2,bal_acc_knn_SMOTE_bl2,precision_knn_SMOTE_bl2, recall_knn_SMOTE_bl2])\n",
    "    \n",
    "    f1_lr_SMOTE_svm,bal_acc_lr_SMOTE_svm,precision_lr_SMOTE_svm, recall_lr_SMOTE_svm,mat_lr_SMOTE_svm=lr(SMOTE_feat_svm,SMOTE_labels_svm,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_SMSVM.append([f1_lr_SMOTE_svm,bal_acc_lr_SMOTE_svm,precision_lr_SMOTE_svm, recall_lr_SMOTE_svm])\n",
    "    f1_svm_SMOTE_svm,bal_acc_svm_SMOTE_svm,precision_svm_SMOTE_svm, recall_svm_SMOTE_svm,mat_svm_SMOTE_svm=svm(SMOTE_feat_svm,SMOTE_labels_svm,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_SMSVM.append([f1_svm_SMOTE_svm,bal_acc_svm_SMOTE_svm,precision_svm_SMOTE_svm, recall_svm_SMOTE_svm])\n",
    "    f1_knn_SMOTE_svm,bal_acc_knn_SMOTE_svm,precision_knn_SMOTE_svm, recall_knn_SMOTE_svm,mat_knn_SMOTE_svm=knn(SMOTE_feat_svm,SMOTE_labels_svm,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_SMSVM.append([f1_knn_SMOTE_svm,bal_acc_knn_SMOTE_svm,precision_knn_SMOTE_svm, recall_knn_SMOTE_svm])\n",
    "    \n",
    "    f1_lr_ADASYN,bal_acc_lr_ADASYN,precision_lr_ADASYN, recall_lr_ADASYN,mat_lr_ADASYN=lr(ADASYN_feat,ADASYN_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_ADA.append([f1_lr_ADASYN,bal_acc_lr_ADASYN,precision_lr_ADASYN, recall_lr_ADASYN])\n",
    "    f1_svm_ADASYN,bal_acc_svm_ADASYN,precision_svm_ADASYN, recall_svm_ADASYN,mat_svm_ADASYN=svm(ADASYN_feat,ADASYN_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_ADA.append([f1_svm_ADASYN,bal_acc_svm_ADASYN,precision_svm_ADASYN, recall_svm_ADASYN])\n",
    "    f1_knn_ADASYN,bal_acc_knn_ADASYN,precision_knn_ADASYN, recall_knn_ADASYN,mat_knn_ADASYN=knn(ADASYN_feat,ADASYN_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_ADA.append([f1_knn_ADASYN,bal_acc_knn_ADASYN,precision_knn_ADASYN, recall_knn_ADASYN])\n",
    "    \n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5428 33 23 185\n",
      "balanced accuracy_LR: 0.941690113814038\n",
      "f1 score_LR: 0.8685446009389671\n",
      "confusion matrix_LR [[5428   33]\n",
      " [  23  185]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 3993 1468 3 205\n",
      "balanced accuracy_SVM: 0.858380843886017\n",
      "f1 score_SVM: 0.21796916533758637\n",
      "confusion matrix_SVM [[3993 1468]\n",
      " [   3  205]]\n",
      "tn, fp, fn, tp: 5369 92 11 197\n",
      "balanced accuracy_KNN: 0.9651343266237515\n",
      "f1 score_KNN: 0.7927565392354124\n",
      "confusion matrix_KNN [[5369   92]\n",
      " [  11  197]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5415 46 29 179\n",
      "balanced accuracy_LR: 0.9260767786964912\n",
      "f1 score_LR: 0.8267898383371826\n",
      "confusion matrix_LR [[5415   46]\n",
      " [  29  179]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4090 1371 1 207\n",
      "balanced accuracy_SVM: 0.8720696934909076\n",
      "f1 score_SVM: 0.23180291153415455\n",
      "confusion matrix_SVM [[4090 1371]\n",
      " [   1  207]]\n",
      "tn, fp, fn, tp: 5373 88 7 201\n",
      "balanced accuracy_KNN: 0.9751159445297424\n",
      "f1 score_KNN: 0.8088531187122736\n",
      "confusion matrix_KNN [[5373   88]\n",
      " [   7  201]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5418 43 25 184\n",
      "balanced accuracy_LR: 0.936254379685793\n",
      "f1 score_LR: 0.8440366972477065\n",
      "confusion matrix_LR [[5418   43]\n",
      " [  25  184]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4081 1380 2 207\n",
      "balanced accuracy_SVM: 0.8688648257456746\n",
      "f1 score_SVM: 0.2305122494432071\n",
      "confusion matrix_SVM [[4081 1380]\n",
      " [   2  207]]\n",
      "tn, fp, fn, tp: 5344 117 10 199\n",
      "balanced accuracy_KNN: 0.9653642312736945\n",
      "f1 score_KNN: 0.758095238095238\n",
      "confusion matrix_KNN [[5344  117]\n",
      " [  10  199]]\n"
     ]
    }
   ],
   "source": [
    "LR_LoRAS=[]\n",
    "SVM_LoRAS=[]\n",
    "KNN_LoRAS=[]\n",
    "for i in range(3):\n",
    "    \n",
    "    features = training_folds_feats[i]\n",
    "    labels= training_folds_labels[i]\n",
    "    label_1=np.where(labels == 1)[0]\n",
    "    label_1=list(label_1)\n",
    "    features_1_trn=features[label_1]\n",
    "    \n",
    "    label_0=np.where(labels == 0)[0]\n",
    "    label_0=list(label_0)\n",
    "    features_0_trn=features[label_0]\n",
    "    \n",
    "    num_samples=30 \n",
    "    shadow=100\n",
    "    sigma=.005\n",
    "    num_RACOS=(len(features_0_trn)-len(features_1_trn))//len(features_1_trn)\n",
    "    num_afcomb=50\n",
    "    nb_list=Neb_grps(features_1_trn, num_samples)\n",
    "    \n",
    "    LoRAS_1=LoRAS_gen(num_samples,shadow,sigma,num_RACOS,num_afcomb)\n",
    "    LoRAS_train=np.concatenate((LoRAS_1,features_0_trn))\n",
    "    LoRAS_labels=np.concatenate((np.zeros(len(LoRAS_1))+1, np.zeros(len(features_0_trn))))\n",
    "    \n",
    "    f1_lr_LoRAS,bal_acc_lr_LoRAS,precision_lr_LoRAS, recall_lr_LoRAS,mat_lr_LoRAS=lr(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_LoRAS.append([f1_lr_LoRAS,bal_acc_lr_LoRAS,precision_lr_LoRAS, recall_lr_LoRAS])\n",
    "    f1_svm_LoRAS,bal_acc_svm_LoRAS,precision_svm_LoRAS, recall_svm_LoRAS,mat_svm_LoRAS=svm(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_LoRAS.append([f1_svm_LoRAS,bal_acc_svm_LoRAS,precision_svm_LoRAS, recall_svm_LoRAS])\n",
    "    f1_knn_LoRAS,bal_acc_knn_LoRAS,precision_knn_LoRAS, recall_knn_LoRAS,mat_knn_LoRAS=knn(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_LoRAS.append([f1_knn_LoRAS,bal_acc_knn_LoRAS,precision_knn_LoRAS, recall_knn_LoRAS])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5426 35 24 184\n",
      "balanced accuracy_LR: 0.9391031510148888\n",
      "f1 score_LR: 0.8618266978922716\n",
      "confusion matrix_LR [[5426   35]\n",
      " [  24  184]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4014 1447 3 205\n",
      "balanced accuracy_SVM: 0.8603035686616991\n",
      "f1 score_SVM: 0.22043010752688172\n",
      "confusion matrix_SVM [[4014 1447]\n",
      " [   3  205]]\n",
      "tn, fp, fn, tp: 5382 79 10 198\n",
      "balanced accuracy_KNN: 0.9687284309720676\n",
      "f1 score_KNN: 0.8164948453608246\n",
      "confusion matrix_KNN [[5382   79]\n",
      " [  10  198]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5417 44 27 181\n",
      "balanced accuracy_LR: 0.9310675876494865\n",
      "f1 score_LR: 0.836027713625866\n",
      "confusion matrix_LR [[5417   44]\n",
      " [  27  181]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4118 1343 1 207\n",
      "balanced accuracy_SVM: 0.8746333265251504\n",
      "f1 score_SVM: 0.2354948805460751\n",
      "confusion matrix_SVM [[4118 1343]\n",
      " [   1  207]]\n",
      "tn, fp, fn, tp: 5375 86 8 200\n",
      "balanced accuracy_KNN: 0.9728952150211994\n",
      "f1 score_KNN: 0.8097165991902834\n",
      "confusion matrix_KNN [[5375   86]\n",
      " [   8  200]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 5412 49 24 185\n",
      "balanced accuracy_LR: 0.9380973742474914\n",
      "f1 score_LR: 0.8352144469525958\n",
      "confusion matrix_LR [[5412   49]\n",
      " [  24  185]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tn, fp, fn, tp: 4114 1347 3 206\n",
      "balanced accuracy_SVM: 0.8694939058955675\n",
      "f1 score_SVM: 0.23382519863791143\n",
      "confusion matrix_SVM [[4114 1347]\n",
      " [   3  206]]\n",
      "tn, fp, fn, tp: 5354 107 11 198\n",
      "balanced accuracy_KNN: 0.9638874700026021\n",
      "f1 score_KNN: 0.7704280155642024\n",
      "confusion matrix_KNN [[5354  107]\n",
      " [  11  198]]\n"
     ]
    }
   ],
   "source": [
    "LR_tLoRAS=[]\n",
    "SVM_tLoRAS=[]\n",
    "KNN_tLoRAS=[]\n",
    "from sklearn.manifold import TSNE\n",
    "for i in range(3):\n",
    "    \n",
    "    features = training_folds_feats[i]\n",
    "    labels= training_folds_labels[i]\n",
    "    label_1=np.where(labels == 1)[0]\n",
    "    label_1=list(label_1)\n",
    "    features_1_trn=features[label_1]\n",
    "    \n",
    "    label_0=np.where(labels == 0)[0]\n",
    "    label_0=list(label_0)\n",
    "    features_0_trn=features[label_0]\n",
    "    \n",
    "    data_embedded_min = TSNE().fit_transform(features_1_trn)\n",
    "    result_min= pd.DataFrame(data = data_embedded_min, columns = ['t-SNE0', 't-SNE1'])\n",
    "    min_t=np.asmatrix(result_min)\n",
    "    min_t=min_t[0:len(features_1_trn)]\n",
    "    min_t=min_t[:, [0,1]]\n",
    "    \n",
    "    num_samples=30 \n",
    "    shadow=100\n",
    "    sigma=.005\n",
    "    num_RACOS=(len(features_0_trn)-len(features_1_trn))//len(features_1_trn)\n",
    "    num_afcomb=50\n",
    "    nb_list=Neb_grps(min_t, num_samples)\n",
    "    \n",
    "    LoRAS_1=LoRAS_gen(num_samples,shadow,sigma,num_RACOS,num_afcomb)\n",
    "    LoRAS_train=np.concatenate((LoRAS_1,features_0_trn))\n",
    "    LoRAS_labels=np.concatenate((np.zeros(len(LoRAS_1))+1, np.zeros(len(features_0_trn))))\n",
    "    \n",
    "    f1_lr_LoRAS,bal_acc_lr_LoRAS,precision_lr_LoRAS, recall_lr_LoRAS,mat_lr_LoRAS=lr(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    LR_tLoRAS.append([f1_lr_LoRAS,bal_acc_lr_LoRAS,precision_lr_LoRAS, recall_lr_LoRAS])\n",
    "    f1_svm_LoRAS,bal_acc_svm_LoRAS,precision_svm_LoRAS, recall_svm_LoRAS,mat_svm_LoRAS=svm(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    SVM_tLoRAS.append([f1_svm_LoRAS,bal_acc_svm_LoRAS,precision_svm_LoRAS, recall_svm_LoRAS])\n",
    "    f1_knn_LoRAS,bal_acc_knn_LoRAS,precision_knn_LoRAS, recall_knn_LoRAS,mat_knn_LoRAS=knn(LoRAS_train,LoRAS_labels,testing_folds_feats[i],testing_folds_labels[i])\n",
    "    KNN_tLoRAS.append([f1_knn_LoRAS,bal_acc_knn_LoRAS,precision_knn_LoRAS, recall_knn_LoRAS])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stats(arr):\n",
    "    x=np.mean(np.asarray(arr), axis = 0)\n",
    "    y=np.std(np.asarray(arr), axis = 0)\n",
    "    return(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1|Balanced Accuracy|precision|recall :: mean|sd\n",
      "Without Oversampling\n",
      "lr: [0.86697083 0.92268866 0.88512568 0.84958901] [0.01060415 0.00341861 0.01539276 0.00624579]\n",
      "svm: [0.86614949 0.91279481 0.90766798 0.82882468] [0.0048657  0.00927121 0.01388145 0.01911944]\n",
      "knn: [0.86908916 0.89768859 0.95602148 0.79678107] [0.00792244 0.00510041 0.01285659 0.01023042]\n",
      "SMOTE Oversampling\n",
      "lr: [0.76449654 0.95442821 0.65086196 0.92802264] [0.02569505 0.00518219 0.03703568 0.0101942 ]\n",
      "svm: [0.1748892  0.81697422 0.09592629 0.99200252] [0.00949262 0.01248916 0.00567663 0.0022556 ]\n",
      "knn: [0.78056364 0.97041168 0.65867719 0.95998957] [0.02190077 0.003221   0.03515312 0.00910366]\n",
      "SMOTE-Bl1 Oversampling\n",
      "lr: [0.74643789 0.95179182 0.62677972 0.92482517] [0.03034254 0.00609206 0.04149589 0.01112581]\n",
      "svm: [0.22916108 0.86762094 0.12960958 0.98879739] [0.00590668 0.00481567 0.00375947 0.00227729]\n",
      "knn: [0.74708726 0.97209861 0.60840924 0.96800239] [0.01062672 0.00114243 0.0144035  0.00222938]\n",
      "SMOTE-Bl2 Oversampling\n",
      "lr: [0.62621174 0.95156078 0.46877798 0.94401761] [0.00985152 0.00870939 0.01316011 0.01930216]\n",
      "svm: [0.22916108 0.86762094 0.12960958 0.98879739] [0.00590668 0.00481567 0.00375947 0.00227729]\n",
      "knn: [0.67428334 0.96524671 0.51865482 0.96479726] [0.0170364  0.00165968 0.0220226  0.00601105]\n",
      "SMOTE-SVM Oversampling\n",
      "lr: [0.75495621 0.9591101  0.63175541 0.93921758] [0.01582183 0.00383004 0.02533111 0.00974874]\n",
      "svm: [0.22693615 0.8670261  0.12814553 0.99200252] [0.00827263 0.0067754  0.00523734 0.0022556 ]\n",
      "knn: [0.77562026 0.97090777 0.65007818 0.96159214] [0.01119517 0.00517256 0.01454527 0.0104109 ]\n",
      "ADASYN Oversampling\n",
      "lr: [0.70324699 0.95339114 0.56412423 0.93443289] [0.02107905 0.00772541 0.02494377 0.01466811]\n",
      "svm: [0.20771223 0.85046901 0.11606491 0.98879739] [0.00805007 0.00782051 0.00498558 0.00227729]\n",
      "knn: [0.70570871 0.97052716 0.5559285  0.97120752] [0.04117373 0.00362076 0.05196972 0.00385991]\n",
      "LoRAS Oversampling\n",
      "lr: [0.84645705 0.93467376 0.8182507  0.87679426] [0.01713201 0.00647136 0.02233495 0.01204667]\n",
      "svm: [0.22676144 0.86643845 0.12804929 0.99039995] [0.00623937 0.00584589 0.00391144 0.00392552]\n",
      "knn: [0.7865683  0.96853817 0.66896982 0.95520488] [0.02117878 0.00465214 0.02830458 0.0081421 ]\n",
      "tLoRAS Oversampling\n",
      "lr: [0.84435629 0.93608937 0.81174179 0.87999172] [0.01235791 0.0035746  0.02089004 0.00693289]\n",
      "svm: [0.22991673 0.8681436  0.13009563 0.98880505] [0.0067426  0.00592751 0.00426114 0.00451656]\n",
      "knn: [0.79887982 0.96850371 0.68776082 0.95360999] [0.02030788 0.00368083 0.02800487 0.00590659]\n"
     ]
    }
   ],
   "source": [
    "print('F1|Balanced Accuracy|precision|recall :: mean|sd')\n",
    "print('Without Oversampling')\n",
    "\n",
    "LR_m, LR_sd=stats(LR)\n",
    "print('lr:',LR_m, LR_sd)\n",
    "SVM_m, SVM_sd=stats(SVM)\n",
    "print('svm:',SVM_m, SVM_sd)\n",
    "KNN_m, KNN_sd= stats(KNN)\n",
    "print('knn:',KNN_m, KNN_sd)\n",
    "\n",
    "print('SMOTE Oversampling')\n",
    "LR_SM_m, LR_SM_sd=stats(LR_SM)\n",
    "print('lr:',LR_SM_m, LR_SM_sd)\n",
    "SVM_SM_m, SVM_SM_sd=stats(SVM_SM)\n",
    "print('svm:',SVM_SM_m, SVM_SM_sd)\n",
    "KNN_SM_m, KNN_SM_sd=stats(KNN_SM)\n",
    "print('knn:',KNN_SM_m, KNN_SM_sd)\n",
    "\n",
    "print('SMOTE-Bl1 Oversampling')\n",
    "LR_SMBL1_m, LR_SMBL1_sd=stats(LR_SMBL1)\n",
    "print('lr:',LR_SMBL1_m, LR_SMBL1_sd)\n",
    "SVM_SMBL1_m,SVM_SMBL1_sd=stats(SVM_SMBL1)\n",
    "print('svm:',SVM_SMBL1_m,SVM_SMBL1_sd)\n",
    "KNN_SMBL1_m,  KNN_SMBL1_sd= stats(KNN_SMBL1)\n",
    "print('knn:',KNN_SMBL1_m,  KNN_SMBL1_sd)\n",
    "\n",
    "print('SMOTE-Bl2 Oversampling')\n",
    "LR_SMBL2_m, LR_SMBL2_sd=stats(LR_SMBL2)\n",
    "print('lr:',LR_SMBL2_m, LR_SMBL2_sd)\n",
    "SVM_SMBL2_m, SVM_SMBL2_sd=stats(SVM_SMBL2)\n",
    "print('svm:',SVM_SMBL2_m, SVM_SMBL2_sd)\n",
    "KNN_SMBL2_m, KNN_SMBL2_sd= stats(KNN_SMBL2)\n",
    "print('knn:',KNN_SMBL2_m, KNN_SMBL2_sd)\n",
    "\n",
    "print('SMOTE-SVM Oversampling')\n",
    "LR_SMSVM_m, LR_SMSVM_sd=stats(LR_SMSVM)\n",
    "print('lr:',LR_SMSVM_m, LR_SMSVM_sd)\n",
    "SVM_SMSVM_m, SVM_SMSVM_sd=stats(SVM_SMSVM)\n",
    "print('svm:',SVM_SMSVM_m, SVM_SMSVM_sd)\n",
    "KNN_SMSVM_m, KNN_SMSVM_sd= stats(KNN_SMSVM)\n",
    "print('knn:',KNN_SMSVM_m, KNN_SMSVM_sd)\n",
    "\n",
    "print('ADASYN Oversampling')\n",
    "LR_ADA_m, LR_ADA_sd=stats(LR_ADA)\n",
    "print('lr:',LR_ADA_m, LR_ADA_sd)\n",
    "SVM_ADA_m, SVM_ADA_sd=stats(SVM_ADA)\n",
    "print('svm:',SVM_ADA_m, SVM_ADA_sd)\n",
    "KNN_ADA_m, KNN_ADA_sd=stats(KNN_ADA)\n",
    "print('knn:',KNN_ADA_m, KNN_ADA_sd)\n",
    "\n",
    "print('LoRAS Oversampling')\n",
    "LR_LoRAS_m, LR_LoRAS_sd=stats(LR_LoRAS)\n",
    "print('lr:',LR_LoRAS_m, LR_LoRAS_sd)\n",
    "SVM_LoRAS_m, SVM_LoRAS_sd=stats(SVM_LoRAS)\n",
    "print('svm:',SVM_LoRAS_m, SVM_LoRAS_sd)\n",
    "KNN_LoRAS_m, KNN_LoRAS_sd=stats(KNN_LoRAS)\n",
    "print('knn:',KNN_LoRAS_m, KNN_LoRAS_sd)\n",
    "\n",
    "print('tLoRAS Oversampling')\n",
    "LR_tLoRAS_m, LR_tLoRAS_sd=stats(LR_tLoRAS)\n",
    "print('lr:',LR_tLoRAS_m, LR_tLoRAS_sd)\n",
    "SVM_tLoRAS_m, SVM_tLoRAS_sd=stats(SVM_tLoRAS)\n",
    "print('svm:',SVM_tLoRAS_m, SVM_tLoRAS_sd)\n",
    "KNN_tLoRAS_m, KNN_tLoRAS_sd=stats(KNN_tLoRAS)\n",
    "print('knn:',KNN_tLoRAS_m, KNN_tLoRAS_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
